import requests
from lxml import etree
import time
import json

# xpath://*[contains(@id,"normalthread")]/tr
class WuAi(object):
    def __init__(self):
        self.url='https://www.52pojie.cn/forum.php?mod=guide&view=newthread'
        self.headers = {
            "User-Agent": "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0) ",
            # "Referer" : "https://www.52pojie.cn/forum.php?mod=guideview=newthread",
            # "Cookie" : "htVD_2132_pc_size_c=0; _uab_collina=154581391104440333253243; Hm_lvt_46d556462595ed05e05f009cdafff31a=1545813919; htVD_2132_saltkey=dJhy5R55; htVD_2132_lastvisit=1545810857; htVD_2132_viewid=tid_844665; htVD_2132_lastact=1545815133%09forum.php%09guide; Hm_lpvt_46d556462595ed05e05f009cdafff31a=1545815605"
        }
    def get_data(self,url):
        response=requests.get(url,headers=self.headers)
        return response.content

    def parse_data(self,data):
        html=etree.HTML(data)
        el_list=html.xpath("//*[contains(@id,'normalthread')]/tr")
        data_list = []
        for el in el_list:
            new_dict={}
            new_dict['标题']=el.xpath('./th/a/text()')[0]
            new_dict['链接']='https://www.52pojie.cn/'+el.xpath('./th/a/@href')[0]
            try:
                new_dict['发布时间']=el.xpath('./td//span[@class="xi1"]/text()')[0]
                data_list.append(new_dict)
            except:
                next_url = None
                return data_list, next_url
        try:
            next_url = 'https://www.52pojie.cn/' + html.xpath('//a[contains(text(),"下一页")]/@href')[0]
        except:
            next_url = None
        return data_list,next_url
    def export(self,data_list):
        # cur_time=time.strftime('%Y-%m-%d', time.localtime(time.time()))
        for data in data_list:
            title = data['标题']
            link = data['链接']
            time = data['发布时间']

            li_str = """
            <li class="list-group-item">
                <a href="%s">
                    <i class="fa fa-file-text"></i>
                    %s
                    <span class="data">%s</span>
                </a>
            </li>
            """ % (link, title, time)
            with open('index.html', "ab") as f:
                f.write(li_str.encode())
    def start_html(self):
        html_str="""<!DOCTYPE html>
<html lang="en">

<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <meta http-equiv="X-UA-Compatible" content="ie=edge">
    <link rel="stylesheet" href="./assets/bootstrap/bootstrap.min.css" />
    <link rel="stylesheet" href="./assets/font-awesome/css/font-awesome.min.css" />
    <link rel="stylesheet" href="./css/index.css">
    <title>Document</title>
</head>

<body>
    <div class="bg">
    <div class="wrap">
        <div class="panel panel-primary">
            <div class="panel-heading  text-center">人气热门</div>
            <ul class="list-group">"""
        with open('index.html', "wb") as f:
            f.write(html_str.encode())
    def end_html(self):
        end_html = """            </ul>
        </div>
    </div>
</div>
</body>

</html>"""
        with open('index.html', "ab") as f:
            f.write(end_html.encode())
    def run(self):
        next_url=self.url
        self.start_html()
        while True:
            list_data=self.get_data(next_url)
            data_list,next_url=self.parse_data(list_data)
            # print(data_list)
            self.export(data_list)
            if not next_url:
                print('爬取结束')
                break
        self.end_html()
        print('已生成HTML文件')

if __name__ == '__main__':
    wu=WuAi()
    wu.run()
